# !/usr/bin/python3
# -*- coding: utf-8 -*-
import random
import time

import requests
from lxml import etree


proxypool_url = 'http://127.0.0.1:5555/random'


def get_random_proxy():
    """
    get random proxy from proxypool
    :return: proxy
    """
    return requests.get(proxypool_url).text.strip()


def request_content(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
            # 'Cookie': 'bid=WTs6MmtLj5o; ll="118267"; douban-fav-remind=1; viewed="35130974"; gr_user_id=f337da85-ed4f-4492-9cca-64ef4a99978c; dbcl2="191999225:5fCdGY2Tdcc"; push_doumail_num=0; push_noty_num=0; __utmv=30149280.19199; __utmz=30149280.1636337891.15.8.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; ck=XrYm; __utmc=30149280; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1637112618%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DdT1h2VkiSWPWaFOA1HZMov-zKrQ8Xl06n0sLwFI7UynPj5eja_qai4AIkoWaff5uD7VkLFJE0plSvGr_fZb9KK%26wd%3D%26eqid%3D876f46d70003022800000006618888d4%22%5D; _pk_ses.100001.8cb4=*; ap_v=0,6.0; __utma=30149280.1874822609.1631851345.1637055284.1637112618.20; __utmt=1; _pk_id.100001.8cb4=2eb745bec83c87ed.1631855534.16.1637112919.1637056343.; __utmb=30149280.16.8.1637112919468',
            'Referer': 'https://www.douban.com/'
        }
        # 从Redis缓存池中取出一定数量的代理
        # proxies = ['1.119.129.2:8080', '115.174.66.148', '113.200.214.164']
        # random.choice(proxies) 随机选取一个
        # 使用代理请求
        # response = requests.get(url, proxy_ip=random.choice(proxies), headers=headers)

        # proxy = get_random_proxy()
        # proxies = {'http': 'http://' + proxy}
        # print(proxies)
        # response = requests.get(url, headers=headers, proxies=proxies)
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        else:
            # 删除当前代理IP,换一个代理IP再次请求
            print('代理IP请求错误')
    except requests.RequestException:
        # 删除当前代理IP,换一个代理IP再次请求
        print('代理IP请求错误')
        return None


def run():
    for i in range(0, 1):
        get_renting(i)
    print('----------end-----------')
    # get_renting_detail('万家丽附近找室友', 'https://www.douban.com/group/topic/240727414/')
    # get_renting_detail('#星沙租房背靠松雅湖 出门逛吾悦 全新装修中建悦和城 一房 拎包入住 情侣首选 房东出远门着急出租 价格私聊',
    # 'https://www.douban.com/group/topic/240722898/')


def write_item_to_file(item):
    # print("开始写入数据======>>>>>" + str(item))
    with open('html/douban_renting_new.html', 'a', encoding='UTF-8') as f:
        f.write(item)
        f.close()


def get_renting(page):
    # https://www.douban.com/group/CSRentRoom/discussion?start=0
    print('------------------', page, '-------------------')
    start = page * 25
    url = 'https://www.douban.com/group/CSRentRoom/discussion?start={}'.format(start)
    print("url", url)

    # url = 'https://www.douban.com/group/274569/'
    html = request_content(url)
    # html = './html/douban_renting_new.html'
    # print('html=', html)
    # write_item_to_file(html)
    if html is not None:
        element = etree.HTML(html)
        # element = etree.parse("./html/douban_renting_new.html", etree.HTMLParser())
        # print(type(element), element, sep='|')
        title = element.xpath('//title/text()')
        # print(type(title), title, sep='|')
        for title_item in title:
            # print(repr(title_item))
            print(title_item.replace('\n', '').replace('\r', '').strip())

        table_tr = element.xpath('//table[@class="olt"]/tr[@class=""]')
        # td_title = element.xpath('//table[@class="olt"]/tr[@class=""]/td[@class="title"]')
        # print(type(table_tr), table_tr, sep='|')
        item_title_text = ''
        item_href_text = ''
        item_author_text = ''
        for table_tr_item in table_tr:
            table_title_item = table_tr_item.xpath('./td[1]/a/@title')
            table_href_item = table_tr_item.xpath('./td[1]/a/@href')
            table_author_item = table_tr_item.xpath('./td[2]/a/text()')
            # print(table_title_item, table_href_item, table_author_item, sep='|')
            for item_title_detail in table_title_item:
                item_title_text = item_title_detail.replace('\n', '').replace('\r', '')
            for item_href_link in table_href_item:
                item_href_text = item_href_link
            for item_author_name in table_author_item:
                item_author_text = item_author_name
            print(item_title_text, item_href_text, item_author_text)
            # print(item_title_text, item_href_text, item_author_text, sep='|')
            # time.sleep(random.randint(5, 10))
            # get_renting_detail(item_title_text, item_href_text)
        print('----------------get_renting-------------------')
        # time.sleep(random.randint(5, 10))
        # get_renting_detail('万家丽附近找室友', 'https://www.douban.com/group/topic/240727414/')
    else:
        print('get_renting None')


def get_renting_detail(title, url):
    # print('title', title, 'url', url)
    html = request_content(url)
    if html is not None:
        # print(html)
        element = etree.HTML(html)
        images = []

        # element = etree.parse("./html/douban_renting_detail.html", etree.HTMLParser())
        # print(type(element), element, sep='|')
        title = element.xpath('//title/text()')
        # print(type(title), title, sep='|')
        for title_item in title:
            print(title_item.replace('\n', '').replace('\r', '').strip())

        topic_doc = element.xpath('//div[@class="topic-doc"]/div[@id="link-report"]/div/div/p/text()')
        # print(topic_doc)
        for topic_doc_text in topic_doc:
            print(topic_doc_text)

        topic_div_img = element.xpath('//div[@class="topic-doc"]/div[@id="link-report"]/div/div/div/div/img/@src')
        for topic_img_src in topic_div_img:
            print(topic_img_src)
            images.append(topic_img_src)
        # print(len(images))
        print('-----------------------------------')
    else:
        print('get_renting_detail None')


if __name__ == '__main__':
    run()
